{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Replication for results in Davidson et al. 2017. \"Automated Hate Speech Detection and the Problem of Offensive Language\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import sys\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import nltk\n",
    "from nltk.stem.porter import *\n",
    "import string\n",
    "import re\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer as VS\n",
    "from textstat.textstat import *\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import LinearSVC\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pickle.load(open(\"../data/labeled_data.p\",'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>hate_speech</th>\n",
       "      <th>offensive_language</th>\n",
       "      <th>neither</th>\n",
       "      <th>class</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!!!!!!!!!!!\"@T_Madison_x: The shit just...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!\"@__BrighterDays: I can not just sit up ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!&amp;#8220;@selfiequeenbri: cause I'm tired of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" &amp;amp; you might not get ya bitch back &amp;amp; ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" @rhythmixx_ :hobbies include: fighting Maria...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" Keeks is a bitch she curves everyone \" lol I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" Murda Gang bitch its Gang Land \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>\" So hoes that smoke are losers ? \" yea ... go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bad bitches is the only thing that i like \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch get up off me \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch nigga miss me with it \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch plz whatever \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch who do you love \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitches get cut off everyday B \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" black bottle &amp;amp; a bad bitch \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" broke bitch cant tell me nothing \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" cancel that bitch like Nino \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" cant you see these hoes wont change \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" fuck no that bitch dont even suck dick \" &amp;#1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" got ya bitch tip toeing on my hardwood floor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>\" her pussy lips like Heaven doors \" &amp;#128524;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" hoe what its hitting for \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" i met that pussy on Ocean Dr . i gave that p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" i need a trippy bitch who fuck on Hennessy \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" i spend my money how i want bitch its my bus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25266</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you ain't gotta be a dyke to like hoes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25267</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you are a hoe, hoe, &amp;amp; a hoe.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25268</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you bitches love yall some corny nigga</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25269</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can masturbate anytime bitch lol &amp;#8220;@g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25270</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can never get a group of hoes together wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25271</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can tell when dick recently been in a puss...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25272</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can't cuff a hoe lmao</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25273</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>you drove me redneck crazy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25274</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you fake niggah lolol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25275</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you got niggas, and i got bitches.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25276</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>you gotta be a new breed of retarded if you do...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25277</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you gotta understand that these bitches are ch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25278</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you hoe spice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25279</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you just want some attention hoe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25280</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>you know what they say, the early bird gets th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25281</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you know what your doing when you favorite a t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25282</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you lil dumb ass bitch, i ain't fuckin wit chu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25283</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you look like AC Green...bitch don't call here...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25284</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you look like your 12 stop talking about fucki...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25285</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you might as well gone pussy pop on a stage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25286</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you niggers cheat on ya gf's? smh....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25287</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you really care bout dis bitch. my dick all in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25288</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you worried bout other bitches, you need me for?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25289</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>you're all niggers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25290</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>you're such a retard i hope you get type 2 dia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25291</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25292</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25294</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25295</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>youu got wild bitches tellin you lies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25296</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24783 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       count  hate_speech  offensive_language  neither class  \\\n",
       "0          3            0                   0        3     2   \n",
       "1          3            0                   3        0     1   \n",
       "2          3            0                   3        0     1   \n",
       "3          3            0                   2        1     1   \n",
       "4          6            0                   6        0     1   \n",
       "5          3            1                   2        0     1   \n",
       "6          3            0                   3        0     1   \n",
       "7          3            0                   3        0     1   \n",
       "8          3            0                   3        0     1   \n",
       "9          3            1                   2        0     1   \n",
       "10         3            0                   3        0     1   \n",
       "11         3            0                   3        0     1   \n",
       "12         3            0                   2        1     1   \n",
       "13         3            0                   3        0     1   \n",
       "14         3            1                   2        0     1   \n",
       "15         3            0                   3        0     1   \n",
       "16         3            0                   3        0     1   \n",
       "17         3            1                   2        0     1   \n",
       "18         3            0                   3        0     1   \n",
       "19         3            0                   3        0     1   \n",
       "20         3            0                   3        0     1   \n",
       "21         3            0                   3        0     1   \n",
       "22         3            0                   3        0     1   \n",
       "23         3            0                   3        0     1   \n",
       "24         3            0                   3        0     1   \n",
       "25         3            0                   2        1     1   \n",
       "26         3            0                   3        0     1   \n",
       "27         3            0                   3        0     1   \n",
       "28         3            0                   3        0     1   \n",
       "29         3            0                   3        0     1   \n",
       "...      ...          ...                 ...      ...   ...   \n",
       "25266      3            1                   2        0     1   \n",
       "25267      3            0                   3        0     1   \n",
       "25268      3            0                   3        0     1   \n",
       "25269      3            0                   3        0     1   \n",
       "25270      3            0                   3        0     1   \n",
       "25271      3            0                   3        0     1   \n",
       "25272      3            0                   3        0     1   \n",
       "25273      3            0                   2        1     1   \n",
       "25274      3            0                   3        0     1   \n",
       "25275      3            1                   2        0     1   \n",
       "25276      3            0                   2        1     1   \n",
       "25277      3            0                   3        0     1   \n",
       "25278      3            0                   3        0     1   \n",
       "25279      3            0                   3        0     1   \n",
       "25280      3            0                   1        2     2   \n",
       "25281      3            0                   3        0     1   \n",
       "25282      3            0                   3        0     1   \n",
       "25283      3            0                   3        0     1   \n",
       "25284      3            0                   3        0     1   \n",
       "25285      3            0                   3        0     1   \n",
       "25286      3            1                   2        0     1   \n",
       "25287      3            0                   3        0     1   \n",
       "25288      3            0                   3        0     1   \n",
       "25289      3            3                   0        0     0   \n",
       "25290      3            2                   1        0     0   \n",
       "25291      3            0                   2        1     1   \n",
       "25292      3            0                   1        2     2   \n",
       "25294      3            0                   3        0     1   \n",
       "25295      6            0                   6        0     1   \n",
       "25296      3            0                   0        3     2   \n",
       "\n",
       "                                                   tweet  \n",
       "0      !!! RT @mayasolovely: As a woman you shouldn't...  \n",
       "1      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...  \n",
       "2      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...  \n",
       "3      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...  \n",
       "4      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...  \n",
       "5      !!!!!!!!!!!!!!!!!!\"@T_Madison_x: The shit just...  \n",
       "6      !!!!!!\"@__BrighterDays: I can not just sit up ...  \n",
       "7      !!!!&#8220;@selfiequeenbri: cause I'm tired of...  \n",
       "8      \" &amp; you might not get ya bitch back &amp; ...  \n",
       "9      \" @rhythmixx_ :hobbies include: fighting Maria...  \n",
       "10     \" Keeks is a bitch she curves everyone \" lol I...  \n",
       "11                    \" Murda Gang bitch its Gang Land \"  \n",
       "12     \" So hoes that smoke are losers ? \" yea ... go...  \n",
       "13         \" bad bitches is the only thing that i like \"  \n",
       "14                               \" bitch get up off me \"  \n",
       "15                       \" bitch nigga miss me with it \"  \n",
       "16                                \" bitch plz whatever \"  \n",
       "17                             \" bitch who do you love \"  \n",
       "18                    \" bitches get cut off everyday B \"  \n",
       "19                    \" black bottle &amp; a bad bitch \"  \n",
       "20                  \" broke bitch cant tell me nothing \"  \n",
       "21                       \" cancel that bitch like Nino \"  \n",
       "22               \" cant you see these hoes wont change \"  \n",
       "23     \" fuck no that bitch dont even suck dick \" &#1...  \n",
       "24     \" got ya bitch tip toeing on my hardwood floor...  \n",
       "25        \" her pussy lips like Heaven doors \" &#128524;  \n",
       "26                          \" hoe what its hitting for \"  \n",
       "27     \" i met that pussy on Ocean Dr . i gave that p...  \n",
       "28        \" i need a trippy bitch who fuck on Hennessy \"  \n",
       "29     \" i spend my money how i want bitch its my bus...  \n",
       "...                                                  ...  \n",
       "25266             you ain't gotta be a dyke to like hoes  \n",
       "25267                   you are a hoe, hoe, &amp; a hoe.  \n",
       "25268             you bitches love yall some corny nigga  \n",
       "25269  you can masturbate anytime bitch lol &#8220;@g...  \n",
       "25270  you can never get a group of hoes together wit...  \n",
       "25271  you can tell when dick recently been in a puss...  \n",
       "25272                          you can't cuff a hoe lmao  \n",
       "25273                         you drove me redneck crazy  \n",
       "25274                              you fake niggah lolol  \n",
       "25275                 you got niggas, and i got bitches.  \n",
       "25276  you gotta be a new breed of retarded if you do...  \n",
       "25277  you gotta understand that these bitches are ch...  \n",
       "25278                                      you hoe spice  \n",
       "25279                   you just want some attention hoe  \n",
       "25280  you know what they say, the early bird gets th...  \n",
       "25281  you know what your doing when you favorite a t...  \n",
       "25282  you lil dumb ass bitch, i ain't fuckin wit chu...  \n",
       "25283  you look like AC Green...bitch don't call here...  \n",
       "25284  you look like your 12 stop talking about fucki...  \n",
       "25285        you might as well gone pussy pop on a stage  \n",
       "25286              you niggers cheat on ya gf's? smh....  \n",
       "25287  you really care bout dis bitch. my dick all in...  \n",
       "25288   you worried bout other bitches, you need me for?  \n",
       "25289                                 you're all niggers  \n",
       "25290  you're such a retard i hope you get type 2 dia...  \n",
       "25291  you's a muthaf***in lie &#8220;@LifeAsKing: @2...  \n",
       "25292  you've gone and broke the wrong heart baby, an...  \n",
       "25294  young buck wanna eat!!.. dat nigguh like I ain...  \n",
       "25295              youu got wild bitches tellin you lies  \n",
       "25296  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...  \n",
       "\n",
       "[24783 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>hate_speech</th>\n",
       "      <th>offensive_language</th>\n",
       "      <th>neither</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.243473</td>\n",
       "      <td>0.280515</td>\n",
       "      <td>2.413711</td>\n",
       "      <td>0.549247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.883060</td>\n",
       "      <td>0.631851</td>\n",
       "      <td>1.399459</td>\n",
       "      <td>1.113299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              count   hate_speech  offensive_language       neither\n",
       "count  24783.000000  24783.000000        24783.000000  24783.000000\n",
       "mean       3.243473      0.280515            2.413711      0.549247\n",
       "std        0.883060      0.631851            1.399459      1.113299\n",
       "min        3.000000      0.000000            0.000000      0.000000\n",
       "25%        3.000000      0.000000            2.000000      0.000000\n",
       "50%        3.000000      0.000000            3.000000      0.000000\n",
       "75%        3.000000      0.000000            3.000000      0.000000\n",
       "max        9.000000      7.000000            9.000000      9.000000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'count', u'hate_speech', u'offensive_language', u'neither', u'class',\n",
       "       u'tweet'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Columns key:\n",
    "count = number of CrowdFlower users who coded each tweet (min is 3, sometimes more users coded a tweet when judgments were determined to be unreliable by CF).\n",
    "\n",
    "\n",
    "hate_speech = number of CF users who judged the tweet to be hate speech.\n",
    "\n",
    "\n",
    "offensive_language = number of CF users who judged the tweet to be offensive.\n",
    "\n",
    "\n",
    "neither = number of CF users who judged the tweet to be neither offensive nor non-offensive.\n",
    "\n",
    "\n",
    "class = class label for majority of CF users.\n",
    "\n",
    "    0 - hate speech\n",
    "    1 - offensive  language\n",
    "    2 - neither\n",
    "\n",
    "tweet = raw tweet text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x11e10ad50>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD5CAYAAADP2jUWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFv5JREFUeJzt3X+QnVV9x/H3bhYI0U1cxuuvFoYi+BnHVtQgCRoklWAI\nyMSBdooKVhAFSdUoM4j8MGGG1lIBjSgiwUwqgjoQEUQDceTHhJWYCqElSr8REKEqdpPZkNUokGT7\nx3N2vF331/Ps3h9wPq8Zhuc59zx7v/fk3s9z7rm/OgYHBzEzs7x0troAMzNrPoe/mVmGHP5mZhly\n+JuZZcjhb2aWIYe/mVmGusbrIGkvYBVwILAPcAnwM2A1MAhsBpZExB5Jy4DjgV3A0ojYKOngifad\n2ptmZmajGTf8gVOAbRFxqqT9gAfTfxdGxN2SrgYWS/olcBQwB9gfWAO8GbiiRN9R9fUNVP5AQk/P\nDPr7d1Y9vGFcVzmuqxzXVU671gWTq61W6+4YqX0iyz43Ahel7Q6Kmfps4J7UthZYAMwD1kXEYEQ8\nAXRJqpXs2xBdXdMa9acnxXWV47rKcV3ltGtd0Jjaxp35R8TvACR1AzcBFwKXRcTQTHwAmAXMBLbV\nHTrU3lGib99odfT0zJjUANRq3ZWPbSTXVY7rKsd1ldOudcHU1zaRZR8k7Q/cDFwVETdI+re6i7uB\n7cCOtD28fU+JvqOazNOxWq2bvr6Bysc3iusqx3WV47rKade6YHK1jXbSGHfZR9LLgXXAJyNiVWre\nJGl+2l4ErAd6gYWSOiUdAHRGxNaSfc3MrAkmMvM/H+gBLpI0tPb/MeALkvYGHgZuiojdktYD91Gc\nVJakvucAKyfY18zMmqDj+fKtnpN5t0+7Pp1zXeW4rnJcVzntWhdMetmn8rt9zMzsBcbhb2aWIYe/\nmVmGHP5mZhma0Pv8zWx0J5xzS8uue9V5b2/Zddvzm2f+ZmYZcvibmWXI4W9mliGHv5lZhhz+ZmYZ\ncvibmWXI4W9mliGHv5lZhhz+ZmYZcvibmWXI4W9mliGHv5lZhhz+ZmYZcvibmWVoQl/pLGkOcGlE\nzJf0TeAV6aIDgQ0RcbKkW4CXAs8Bf4iIRZIOBlYDg8BmYElE7JG0DDge2AUsjYiNU3mjzMxsbOOG\nv6RzgVOB3wNExMmpvQe4C/h46noI8LqIqP+h9SuACyPibklXA4sl/RI4CpgD7A+sAd48NTfHzMwm\nYiLLPo8CJ47QfjFwZUT8RtLLgZcA35V0r6R3pj6zgXvS9lpgATAPWBcRgxHxBNAlqTapW2FmZqWM\nO/OPiDWSDqxvk/Qy4Gj+NOvfG7gcWAHsB/RK2gh01D0TGABmATOBbXV/bqi9b6w6enpm0NU1bbxy\nR1WrdVc+tpFcVzntWlerjDce7Tperqu8qa6t6s84/h1wQ0TsTvtPAVdHxC7gfyVtAgTsqTumG9gO\n7Ejbw9vH1N+/s2KpxaD19Q1UPr5RXFc57VpXK401Hu06Xq6rvMnUNtpJo+q7fRZQLOPU798IIOnF\nwF8DDwObJM1PfRYB64FeYKGkTkkHAJ0RsbViHWZmVkHV8Bfw2NBORKwFtkjaAKwDzk+Bfg5wsaT7\nKJaGboqI+ylOAvdRvNi7ZBL1m5lZBRNa9omIx4G5dfuvG6HP0hHatlC8s2d4+3Jg+cTLNDOzqeQP\neZmZZcjhb2aWIYe/mVmGHP5mZhly+JuZZcjhb2aWIYe/mVmGHP5mZhly+JuZZcjhb2aWIYe/mVmG\nHP5mZhly+JuZZcjhb2aWIYe/mVmGHP5mZhly+JuZZcjhb2aWIYe/mVmGJvQbvpLmAJdGxHxJbwRu\nA36eLv5yRHxL0jLgeGAXsDQiNko6GFgNDAKbgSURsWekvlN6q8zMbEzjhr+kc4FTgd+nptnAFRFx\neV2fN1H8UPscYH9gDfBm4Argwoi4W9LVwGJJvxylr5mZNclEZv6PAicC16X92YAkLaaY/S8F5gHr\nImIQeEJSl6Ra6ntPOm4t8A4gRuobEX1TdqvMzGxM44Z/RKyRdGBd00bg2oi4X9IFwDJgO7Ctrs8A\nMAvoSCFf3zZzlL5jhn9Pzwy6uqaNV+6oarXuysc2kusqp13rapXxxqNdx8t1lTfVtU1ozX+YmyNi\n+9A2cCVwC1BfWTfFCWHPCG07Ruk7pv7+nRVKLdRq3fT1DVQ+vlFcVzntWlcrjTUe7Tperqu8ydQ2\n2kmjyrt97pB0eNo+Grgf6AUWSuqUdADQGRFbgU2S5qe+i4D1Y/Q1M7MmqTLz/zBwpaTngKeAD0XE\nDknrgfsoTihLUt9zgJWS9gYeBm6KiN2j9DUzsyaZUPhHxOPA3LT9APDWEfosB5YPa9tC8c6ecfua\nmVnz+ENeZmYZcvibmWXI4W9mliGHv5lZhhz+ZmYZcvibmWXI4W9mliGHv5lZhhz+ZmYZcvibmWXI\n4W9mliGHv5lZhhz+ZmYZcvibmWXI4W9mliGHv5lZhhz+ZmYZcvibmWXI4W9mlqEJ/YavpDnApREx\nX9IbgCuB3cAzwPsi4reSVgDzgIF02GJgL+AGYF/g18BpEbFT0geBM4FdwCURcdtU3igzMxvbuDN/\nSecC1wLTU9MK4CMRMR/4NvDJ1D4bWBgR89N/TwOfBm6IiCOBTcCZkl4BfJTiR+AXAp+RtM8U3iYz\nMxvHRJZ9HgVOrNs/OSIeTNtdwB8ldQKHANdI6pV0erp8HnB72l4LLAAOB3oj4pl0gngEeP0kb4eZ\nmZUw7rJPRKyRdGDd/m8AJL0F+CfgbcCLKJaCrgCmAXdJ+gkwE3g6HToAzBrWVt8+pp6eGXR1TRv/\nFo2iVuuufGwjua5y2rWuVhlvPNp1vFxXeVNd24TW/IeT9A/ABcDxEdEnaRqwIiJ2psvvBA4FdgDd\nwB/S/7fXtQ0Zah9Tf//OKqUCxaD19Q2M37HJXFc57VpXK401Hu06Xq6rvMnUNtpJo/S7fSSdQjHj\nnx8Rj6Xm1wC9kqZJ2otiuecBoBc4LvVZBKwHNgJHSpouaRbwWmBz2TrMzKy6UuGfZvhfoJitf1vS\n3ZIujoiHgeuADcA9wNci4qfAJcDJknqBI4AvRsRT6W+sB+4ELoiIP07ZLTIzs3FNaNknIh4H5qbd\n/Ubp81ngs8PafgscO0LflcDKMoWamdnU8Ye8zMwy5PA3M8uQw9/MLEMOfzOzDDn8zcwy5PA3M8uQ\nw9/MLEMOfzOzDDn8zcwy5PA3M8uQw9/MLEMOfzOzDDn8zcwy5PA3M8uQw9/MLEMOfzOzDDn8zcwy\n5PA3M8uQw9/MLEMT+g1fSXOASyNivqSDgdXAILAZWBIReyQtA44HdgFLI2Jjmb5TfLvMzGwM4878\nJZ0LXAtMT01XABdGxJFAB7BY0puAo4A5wMnAlyr0NTOzJpnIss+jwIl1+7OBe9L2WmABMA9YFxGD\nEfEE0CWpVrKvmZk1ybjLPhGxRtKBdU0dETGYtgeAWcBMYFtdn6H2Mn37xqqjp2cGXV3Txit3VLVa\nd+VjG8l1ldOudbXKeOPRruPlusqb6tomtOY/zJ667W5gO7AjbQ9vL9N3TP39OyuUWqjVuunrG6h8\nfKO4rnLata5WGms82nW8XFd5k6lttJNGlXf7bJI0P20vAtYDvcBCSZ2SDgA6I2Jryb5mZtYkVWb+\n5wArJe0NPAzcFBG7Ja0H7qM4oSyp0NfMzJpkQuEfEY8Dc9P2Fop36wzvsxxYPqxtwn3NzKx5/CEv\nM7MMOfzNzDLk8Dczy5DD38wsQw5/M7MMOfzNzDLk8Dczy5DD38wsQw5/M7MMOfzNzDLk8Dczy5DD\n38wsQw5/M7MMOfzNzDLk8Dczy5DD38wsQw5/M7MMOfzNzDLk8Dczy1CVH3BH0vuB96fd6cAbgHcD\nlwFPpvZlwHrgKuBQ4BngjIh4RNJcYAWwC1gXERdXrN/MzCqoFP4RsRpYDSDpS8AqYDZwbkSsGeon\n6URgekQckQL/cmAxcDVwEvAY8D1Jb4yITZO4HWZmVsKkln0kHQa8LiKuoQj/0yWtl3S5pC5gHnA7\nQERsAA6TNBPYJyIejYhB4A5gwaRuhZmZlVJp5l/nfGBoyeYHwHeAX1DM7M8CZgJP1/Xfndp21LUN\nAAeNd0U9PTPo6ppWudBarbvysY3kuspp17paZbzxaNfxcl3lTXVtlcNf0ksARcRdqWlVRGxPl91C\nsazzNFBfcSdF8Ne3dQPbx7u+/v6dVUulVuumr2+g8vGN4rrKade6Wmms8WjX8XJd5U2mttFOGpNZ\n9nkb8EMASR3Af0n6y3TZ0cD9QC9wXOozF3goInYAz0p6dTpuIcULw2Zm1iSTWfYRxQu2RMSgpDOA\nb0v6A/AzYCXFMs8xkn4EdACnpWPPAq4HplG82+fHk6jDzMxKqhz+EfHZYfvrgHUjdD1rhGM3AHOr\nXreZmU2OP+RlZpYhh7+ZWYYc/mZmGXL4m5llyOFvZpYhh7+ZWYYc/mZmGXL4m5llyOFvZpYhh7+Z\nWYYc/mZmGXL4m5llyOFvZpYhh7+ZWYYc/mZmGXL4m5llyOFvZpYhh7+ZWYYc/mZmGar8G76SHgB2\npN1fAF8BVgC7KH6U/WJJncBVwKHAM8AZEfGIpLnD+07iNpiZWUmVwl/SdKAjIubXtT0InAQ8BnxP\n0huBvwKmR8QRKfAvBxYDVw/vGxGbJnVLzMxswqrO/A8FZkhal/7GcmCfiHgUQNIdwALglcDtABGx\nQdJhkmaO0tfhb2bWJFXDfydwGXAtcAiwFthed/kAcBAwE3i6rn13atsxQt8x9fTMoKtrWsVyoVbr\nrnxsI7muctq1rlYZbzzadbxcV3lTXVvV8N8CPBIRg8AWSU8D+9Vd3k1xMpiRtod0UgR/9wh9x9Tf\nv7NiqcWg9fUNVD6+UVxXOe1aVyuNNR7tOl6uq7zJ1DbaSaPqu31Op1i/R9KrKEL+95JeLakDWAis\nB3qB41K/ucBDEbEDeHaEvmZm1iRVZ/5fBVZLuhcYpDgZ7AGuB6ZRvIPnx5L+AzhG0o+ADuC0dPxZ\nw/tO4jaYmVlJlcI/Ip4F3jPCRXOH9dtDEfTDj98wvK+ZmTWPP+RlZpYhh7+ZWYYc/mZmGXL4m5ll\nqPJ3+5iZ5eT0f72zZdf93csXT/nf9MzfzCxDDn8zsww5/M3MMuTwNzPLkMPfzCxDDn8zsww5/M3M\nMuTwNzPLkMPfzCxDDn8zsww5/M3MMuTwNzPLkMPfzCxDDn8zswxV+kpnSXsBq4ADgX2AS4AngduA\nn6duX46Ib0laBhwP7AKWRsRGSQcDqyl+/H0zsCT93q+ZmTVB1Zn/KcC2iDgSOBb4IjAbuCIi5qf/\nviXpTcBRwBzgZOBL6fgrgAvT8R3A1H9ZtZmZjarqj7ncCNyUtjsoZvWzAUlaTDH7XwrMA9ZFxCDw\nhKQuSbXU9550/FrgHcDNFWsxM7OSKoV/RPwOQFI3xUngQorln2sj4n5JFwDLgO3AtrpDB4BZQEc6\nIdS3jamnZwZdXdOqlAtArdZd+dhGcl3ltGtdrTLeeLTreLmu8qa6tso/4yhpf4rZ+lURcYOkl0TE\n9nTxzcCVwC1AfcXdFCeEPSO0jam/f2fVUqnVuunrG6h8fKO4rnLata5WGms82nW8XFc1VWsb7aRR\nac1f0suBdcAnI2JVar5D0uFp+2jgfqAXWCipU9IBQGdEbAU2SZqf+i4C1lepw8zMqqk68z8f6AEu\nknRRavsE8DlJzwFPAR+KiB2S1gP3UZxolqS+5wArJe0NPMyfXj8wM7MmqLrm/zHgYyNc9NYR+i4H\nlg9r20LxLiAzM2sBf8jLzCxDDn8zsww5/M3MMuTwNzPLkMPfzCxDDn8zswxV/oTv88kJ59zSsute\ndd7bW3bdZmaj8czfzCxDDn8zsww5/M3MMuTwNzPLkMPfzCxDDn8zsww5/M3MMuTwNzPLkMPfzCxD\nDn8zsww5/M3MMuTwNzPLUMu+2E1SJ3AVcCjwDHBGRDzSqnrMzHLSypn/u4DpEXEEcB5weQtrMTPL\nSivDfx5wO0BEbAAOa2EtZmZZ6RgcHGzJFUu6FlgTEWvT/hPAQRGxqyUFmZllpJUz/x1Ad91+p4Pf\nzKw5Whn+vcBxAJLmAg+1sBYzs6y08mccbwaOkfQjoAM4rYW1mJllpWVr/mZm1jr+kJeZWYYc/mZm\nGXL4m5llqJUv+E6J8b4mQtIHgTOBXcAlEXGbpJcCNwD7Ar8GTouInU2u6+PAyWn3+xFxsaQO4H+A\nn6f2+yLiU02uawXFB/AGUtNiYC9aOF6S3gB8vq77XIpPiG8EtgCbU/vNEbFiKuuqq28OcGlEzB/W\nfgLwaYr716qIWClpX+DrwMsoxvEfI6KvyXW9G1ia6noIODsi9kh6gOJt1gC/iIiGvNFijLo+DpwB\nDI3HmcATtHC8JL0C+GZdtzdQfOvAV2j843EvYBVwILAPRUbdWnd5w+5fz/vwp+5rItJbRi+nCKyh\nf9SPUnx6eDpwr6QfUAzmDRGxWtJ5FHfAzzWxroOA9wJzgD2prpuBncADEXHCFNcyobqS2cDCiNg6\n1CDpC7RwvCLiQWB+quXvgV9FxO2SFgDfiIiPTHEt/4+kc4FTgd8Pa9+LYhzenC7rlXQrxb/tQxGx\nXNLJwIXAx5pY177AJcDfRMROSd8A3ilpHdAxPJCbVVcyG3hfRNxf1/8TtHC8IuIp/nT/OgL4Z2Al\n8Goa/3g8BdgWEadK2g94ELg11dLQ+9cLYdlnrK+JOBzojYhnIuJp4BHg9fXHAGuBBU2u60ng2IjY\nHRGDFDPrP1I8MP5C0l2Svi9Jzawrzb4PAa6R1Cvp9OHH0JrxGqrvRcDF/OmOPhuYLekeSTdKemUD\n6gJ4FDhxhPbXAo9ERH9EPAvcC7yN5ozXWHU9A7yl7tlZF8X961BghqR1ku5MJ9lm1gXFv9mnJN0r\naWgW3erxAiA9874S+HBE7KY5j8cbgYvSdgfFDH9IQ+9fL4Twnwk8Xbe/W1LXKJcNALOGtQ+1Na2u\niHguIrZK6pB0GbApIrYAvwE+ExF/C/wLxVO7ptUFvIjizn8KcCxwtqTX0+LxqvMB4Ma6ZyX/DXw6\nIo4CvpNqn3IRsQZ4boSLWnn/GrWuiNgTEb8FkPQR4MXADyieWV4GLATOAq4fYYwbVlfyzXTdbwfm\nSXonLR6vOicAP42ISPsNfzxGxO8iYkBSN3ATxSx+SEPvXy+E8B/rayKGX9YNbB/WPtTWzLqQNB24\nPvU5OzX/BLgFICLuBV6VZiPNqmsnsCIidkbEAHAnxWyx5eOVvBe4tm7/TuCutH0z8MYG1DWWVt6/\nxiSpM00sjgFOSs8wtwBfj4jBNNnYBjTq2dJINXUAn4+IrWkm+z2Kf7OWj1dyCnBN3X4zHo9I2p/i\nfnxdRNxQd1FD718vhPAf62siNgJHSpouaRbF06jN9ccAi4D1zawr3YFuAf4zIs5MTzEBllG8SIek\nQ4En04O2KXUBr6FYV5yW1hvnAQ/Q4vFKbbOAfSLiybrma4GT0vbRwP0018PAIZL2k7Q3xVPy+2jO\neI3nKxSvc72rbvnndNJXp0t6FcUM8jdNrGkmsFnSi9Nj4O0U/2btMF5QLDX+qG6/4Y9HSS8H1gGf\njIhVwy5u6P3rhfCC7599TUR6AemRiLg1vVi5nuJEd0FE/FHSJcC/p3cCbQXe08y6gGnAUcA+khal\n/p8C/hX4uqTjKdb+3t/MutJ4XQdsoHh6/LWI+Gmrxyu9++E1wOPDjjkPWCXpbIoXxM5oQF1/RtJ7\ngBdHxDWpxjso7l+rIuJXkr5MMV73As/SmPEatS6KGesHKO73d6al6hXAV4HVqa5B4PRmfJnisPE6\nn2KW+wzww4j4vqS7aeF4pbpqwI5h4d6Mx+P5QA9wkaShtf+VwIsaff/y1zuYmWXohbDsY2ZmJTn8\nzcwy5PA3M8uQw9/MLEMOfzOzDDn8zcwy5PA3M8vQ/wHCSsqCkthQxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11e670750>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['class'].hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This histogram shows the imbalanced nature of the task - most tweets containing \"hate\" words as defined by Hatebase were \n",
    "only considered to be offensive by the CF coders. More tweets were considered to be neither hate speech nor offensive language than were considered hate speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tweets=df.tweet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Feature generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "stopwords=stopwords = nltk.corpus.stopwords.words(\"english\")\n",
    "\n",
    "other_exclusions = [\"#ff\", \"ff\", \"rt\"]\n",
    "stopwords.extend(other_exclusions)\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "\n",
    "def preprocess(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, '', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, '', parsed_text)\n",
    "    return parsed_text\n",
    "\n",
    "def tokenize(tweet):\n",
    "    \"\"\"Removes punctuation & excess whitespace, sets to lowercase,\n",
    "    and stems tweets. Returns a list of stemmed tokens.\"\"\"\n",
    "    tweet = \" \".join(re.split(\"[^a-zA-Z]*\", tweet.lower())).strip()\n",
    "    tokens = [stemmer.stem(t) for t in tweet.split()]\n",
    "    return tokens\n",
    "\n",
    "def basic_tokenize(tweet):\n",
    "    \"\"\"Same as tokenize but without the stemming\"\"\"\n",
    "    tweet = \" \".join(re.split(\"[^a-zA-Z.,!?]*\", tweet.lower())).strip()\n",
    "    return tweet.split()\n",
    "\n",
    "vectorizer = TfidfVectorizer(\n",
    "    tokenizer=tokenize,\n",
    "    preprocessor=preprocess,\n",
    "    ngram_range=(1, 3),\n",
    "    stop_words=stopwords,\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=10000,\n",
    "    min_df=5,\n",
    "    max_df=0.75\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Construct tfidf matrix and get relevant scores\n",
    "tfidf = vectorizer.fit_transform(tweets).toarray()\n",
    "vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names())}\n",
    "idf_vals = vectorizer.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Get POS tags for tweets and save as a string\n",
    "tweet_tags = []\n",
    "for t in tweets:\n",
    "    tokens = basic_tokenize(preprocess(t))\n",
    "    tags = nltk.pos_tag(tokens)\n",
    "    tag_list = [x[1] for x in tags]\n",
    "    tag_str = \" \".join(tag_list)\n",
    "    tweet_tags.append(tag_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#We can use the TFIDF vectorizer to get a token matrix for the POS tags\n",
    "pos_vectorizer = TfidfVectorizer(\n",
    "    tokenizer=None,\n",
    "    lowercase=False,\n",
    "    preprocessor=None,\n",
    "    ngram_range=(1, 3),\n",
    "    stop_words=None,\n",
    "    use_idf=False,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=5000,\n",
    "    min_df=5,\n",
    "    max_df=0.75,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Construct POS TF matrix and get vocab dict\n",
    "pos = pos_vectorizer.fit_transform(pd.Series(tweet_tags)).toarray()\n",
    "pos_vocab = {v:i for i, v in enumerate(pos_vectorizer.get_feature_names())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Now get other features\n",
    "sentiment_analyzer = VS()\n",
    "\n",
    "def count_twitter_objs(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "    4) hashtags with HASHTAGHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned.\n",
    "    \n",
    "    Returns counts of urls, mentions, and hashtags.\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    hashtag_regex = '#[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, 'URLHERE', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, 'MENTIONHERE', parsed_text)\n",
    "    parsed_text = re.sub(hashtag_regex, 'HASHTAGHERE', parsed_text)\n",
    "    return(parsed_text.count('URLHERE'),parsed_text.count('MENTIONHERE'),parsed_text.count('HASHTAGHERE'))\n",
    "\n",
    "def other_features(tweet):\n",
    "    \"\"\"This function takes a string and returns a list of features.\n",
    "    These include Sentiment scores, Text and Readability scores,\n",
    "    as well as Twitter specific features\"\"\"\n",
    "    sentiment = sentiment_analyzer.polarity_scores(tweet)\n",
    "    \n",
    "    words = preprocess(tweet) #Get text only\n",
    "    \n",
    "    syllables = textstat.syllable_count(words)\n",
    "    num_chars = sum(len(w) for w in words)\n",
    "    num_chars_total = len(tweet)\n",
    "    num_terms = len(tweet.split())\n",
    "    num_words = len(words.split())\n",
    "    avg_syl = round(float((syllables+0.001))/float(num_words+0.001),4)\n",
    "    num_unique_terms = len(set(words.split()))\n",
    "    \n",
    "    ###Modified FK grade, where avg words per sentence is just num words/1\n",
    "    FKRA = round(float(0.39 * float(num_words)/1.0) + float(11.8 * avg_syl) - 15.59,1)\n",
    "    ##Modified FRE score, where sentence fixed to 1\n",
    "    FRE = round(206.835 - 1.015*(float(num_words)/1.0) - (84.6*float(avg_syl)),2)\n",
    "    \n",
    "    twitter_objs = count_twitter_objs(tweet)\n",
    "    retweet = 0\n",
    "    if \"rt\" in words:\n",
    "        retweet = 1\n",
    "    features = [FKRA, FRE,syllables, avg_syl, num_chars, num_chars_total, num_terms, num_words,\n",
    "                num_unique_terms, sentiment['neg'], sentiment['pos'], sentiment['neu'], sentiment['compound'],\n",
    "                twitter_objs[2], twitter_objs[1],\n",
    "                twitter_objs[0], retweet]\n",
    "    #features = pandas.DataFrame(features)\n",
    "    return features\n",
    "\n",
    "def get_feature_array(tweets):\n",
    "    feats=[]\n",
    "    for t in tweets:\n",
    "        feats.append(other_features(t))\n",
    "    return np.array(feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "other_features_names = [\"FKRA\", \"FRE\",\"num_syllables\", \"avg_syl_per_word\", \"num_chars\", \"num_chars_total\", \\\n",
    "                        \"num_terms\", \"num_words\", \"num_unique_words\", \"vader neg\",\"vader pos\",\"vader neu\", \\\n",
    "                        \"vader compound\", \"num_hashtags\", \"num_mentions\", \"num_urls\", \"is_retweet\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "feats = get_feature_array(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Now join them all up\n",
    "M = np.concatenate([tfidf,pos,feats],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24783, 11172)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Finally get a list of variable names\n",
    "variables = ['']*len(vocab)\n",
    "for k,v in vocab.iteritems():\n",
    "    variables[v] = k\n",
    "\n",
    "pos_variables = ['']*len(pos_vocab)\n",
    "for k,v in pos_vocab.iteritems():\n",
    "    pos_variables[v] = k\n",
    "\n",
    "feature_names = variables+pos_variables+other_features_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Running the model\n",
    "\n",
    "The best model was selected using a GridSearch with 5-fold CV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = pd.DataFrame(M)\n",
    "y = df['class'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "select = SelectFromModel(LogisticRegression(class_weight='balanced',penalty=\"l1\",C=0.01))\n",
    "X_ = select.fit_transform(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model = LinearSVC(class_weight='balanced',C=0.01, penalty='l2', loss='squared_hinge',multi_class='ovr').fit(X_, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model = LogisticRegression(class_weight='balanced',penalty='l2',C=0.01).fit(X_,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_preds = model.predict(X_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Evaluating the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report = classification_report( y, y_preds )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.44      0.60      0.51      1430\n",
      "          1       0.97      0.91      0.94     19190\n",
      "          2       0.81      0.95      0.88      4163\n",
      "\n",
      "avg / total       0.91      0.90      0.90     24783\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUAAAAFACAYAAADNkKWqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtsVNe9L/Cv29xKtMFjhpNcCUxim9BfgTQNYBJIlYaD\nH+QlkYIN7W2SljSGtoraSgkx3Oq0TW7PAcNp0qhqg3GUNGl6erENaqSqPGwIbVUaYmwIKqS/GzBu\ncKKrRDb2EAXpKuncP9ba43nae8PMnhnv70ey8N6z9uzljefn9V4l0WgURERB9Il8Z4CIKF8YAIko\nsBgAiSiwGACJKLAYAIkosBgAiSiwrsp3BtxYt24dx+r4ZOXKlfnOQiDcfvvt+c5CYEyfPr0k02ss\nARJRYDEAElFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFgMQAS\nUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFg\nMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYDEA\nElFgMQASUWAxABJRYF2V7wwEwfXXX4/33nsPly5dyndWiNI6dOgQ3n33XQCAiGDx4sWu0/zud78D\nALzzzjuora2FiPiU6yvHAHgZqqurcc011wAA/vGPf+D06dMZ082fPx9/+MMfYsHP7bVknDx5EsPD\nwwCAmTNnYs6cOSlpjh49iqNHj+LSpUu4//77MXPmzNhrly5dwuDgYNrryLh48SJ+/etf44UXXgAA\nNDQ0oLOz01Wanp4evPrqq3jmmWcAACtWrMD+/fv9/QGuAKvAHk2ZMgX33HMP9u7di7179+LrX/96\n2nR33HEHli1bhhdffBHvv/++p2vJuHTpEg4fPoxly5Zh2bJl2LNnT0qaoaEhlJeX47vf/S5uvfVW\nvPrqqwmvtbW14eTJk35mu+gcPHgQn/vc52LHM2fOjJXqJkrz+uuv47Of/Wzs/IwZM9DT05P7TGcJ\nA6BHt9xyCwYGBmLH77//Pu64446ENFOmTME3vvENvPjii56vpTEnT55MKM2Fw2EcPXo0Ic306dNj\nacLhMMLhcMJrN910kz+ZLWKqmvCcZ8yYgXfeecdVmg8++CBWLQaA0tLSlGsLmW8BUERKReRZEdli\nj5tEpNSv+2dLRUUF3nvvvdjx+++/j2uvvTYhzS233AIAWLZsGX70ox/FSnpurqUxg4ODmD59euw4\nHA5jaGgobdqhoSGcPHkSd999t1/ZmzTiAxgAXH311SnnMqVZvHgxjh07Fjv/zjvvoLS0eD7WfpYA\nOwGUOAeq2gagw8f7Z4XTfuf48MMPU85de+21OHz4MDo6OvDEE09g3rx5mDdvnqtraYzT9ueYMmVK\nyjnH0aNHcebMGbS1tfmRtUllxowZCccffPBByrlMaZYvX44HHngAP/jBD/Dyyy/j4sWLRdUJ4mcA\n7FLVbwHojzuX2tVU4Jz2PMenP/3plHOACW6O06dPY/78+a6vJSO+OguYNsHkc467774bzc3NOHPm\nDHvbPRKRhGrru+++i7lz57pOc//99+Pf//3fISKYMWNGQlW50PnaBigiDwOoEpFVItIDoNvP+2fD\nwMBAQrX1mmuuwblz5xLSvPfeewkluw8//BAffPCBq2tpTHl5eUKVd3h4GOXl5RnTT5kyBTNnzsSU\nKVP8yN6kUVNTg7///e+x40gkguXLlwMwQ18mSgOYXuJf/vKX+MlPfuJTrrPDtwCoqtsBXAAwDcBX\nALQDeNiv+2fL66+/joqKitjxZz7zmVgbSHV1dSzN9ddfH0tz/fXX4/Tp0+NeS6luuummhFLHpUuX\nYp0amXp2M5UQKbOpU6figQcewMsvv4xf/OIX+M53vgPABLWWlpZx0wAmSDrBr5hKfwBQEo1GfbmR\niDysqs8lnXtWVb890bXr1q3zJ5MuOWP5rr76apw6dQqnT5/GlClTsH37djzyyCMAgHnz5mHx4sWx\nXt8//vGPGa8tJCtXrsx3FhI44wA//PBDzJkzB3PmzMGlS5fQ0tKCH//4x3jrrbewZ88e3HrrrQiH\nw5gzZ06sBDg0NITf/OY3AICvfe1rCR0q+Xb77bfnOwuBMX369JJMr+U8AIrIAgBtACoBxLdghwH0\nq+qE7YCFFgAns0ILgJMVA6B/xguAOZ8JoqrHRaQJQDVS2/zSj2kgIvKBL1PhVPU4gOMishxAVdxL\nG1CEPcFENDn4NhdYRNphqsBVMENhqgDs8uv+RETJ/BwGs8uOA2xR1W+paj2A2T7en4gogZ+rwSwW\nkVpV/bYtDQ4BqPXx/kRECfwcB7gJwE572ARTDV7j1/2JiJLlNAAmL3ZgO0OgqqN2YPTZXN6fiGg8\nua4Cd4jIDgCjaV4rg+kFXpHjPBARpZXrAFiHxGEuIYwFwzIAHOBMRHmT6wC4wS57BQAQkR22J9g5\nbsrx/YmIMsppG2B88LOiE7xOROSbXHeC3DzB6xW5vD8R0Xj86ATpxNic32oRecx+Px1mHCCnwhFR\nXuQ6AM6G6QhxVoG5AKDefh+GWSGGiCgvct4LrKoHM70oIjU5vj8RUUa57gTJGPzcvE5ElEvcF5iI\nAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKL\nAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQ\niAKLAZCIAosBkIgCqyQajbpKKCL7AYwAaAKwBkCzfWm9qr6am+wZly5dcpdJumLXXnttvrMQCEND\nQ/nOQmB86lOfKsn02lUe3mc2gIUApgPYCSAKYJP9ymkAJCLKBS9V4A5VjQDogAl+G1R1O4C+nOSM\niCjHvATARSIyBFMK7FPV50SkEsD63GSNiCi3vFSBG2GCXRRAm4gsALAZwMFcZIyIKNdclwBVdRRA\nGMBsVR1V1eMAulR1Tc5yR0SUQ64DoIi0w/T8VsWdPiYiW7KeKyIiH3hpAzyrqp8AcDzuXBhsAySi\nIuWlDXCRiDwMoFJEHgOwGEADgP6c5IyIKMe8BMCdANphOkEaADiDC5szXkFEVMBczwQBABGpArAB\nQCVMya9VVc/lKG8xnAniH84E8QdngvhnvJkgngJgOiKyXFUPXdGbTIAB0D8MgP5gAPTPZU+FE5Ed\nAJYDqAaQqaQXmuh9iIgK0USBazHMHODpAKZlSMPSGREVpXEDoKouEpGQqo6KSLOd+5tARDbmLntE\nRLkz4ThAOwMEABaLyP9J83pKUCQiKgaeFkMAMDphKiKiIuGl82I9gFoRKU0636Kq385inoiIfOEl\nAB6w/z6e5jUGQCIqOl4CYAlSp72VAKjIWm6IiHzkJQDWqWrK2n8iUpPF/BAR+cbrVLhSmA2RZgM4\ng7Fl8nOKM0H8w5kg/uBMEP9kZSqcXf6+F2bmh/OGFwAsV9U3rjST42EA9A8DoD8YAP2TrV3hWgF0\nAxgGcBZmdshsANsArLiSDBIR5YOXANjFmSBENJl46gSx1eB+mA3Sy2BKgFUAOBuEiIqOlwC4CcAx\njC1+UGK/X5TtTBER+cF1AFTVPhFx9gCpgmkH3Bk3V5iIqKi4DoAiskVVN8N0esSfr4BZFv+sqv5n\ndrNHRJQ7XhZD2CAiQyLysYi8LiJT7flOmFLhv9jNkoiIioKXAFgG0+53HMANAFrs+YUA+lR1E4B/\nyW72iIhyx0sA3KmqYVWtVtUw0i+NxdGdRFQ0vPQCV4vIf8AMhJ4NszRWT1KacNZyRkSUY173Bd5h\nv4/CbJTkLI5QIiL7YcYHEhEVBddVYFXdCaAOZl/gsKoeh9kfuFFVq2GmyW3KSS6JiHLA63aWtQDK\nVLXNHjeq6nMA9wYhouLjugQoIu0w4/2q4k73isiWrOeKiMgHXnqBz6rqJ2CGwTicmSFEREXHSxV4\nkYg8DKDSDnheDKABqcvkk0uRSASlpcl7TFG8lStXorKyEgBw4sQJHD58OCXNunXrAACVlZXYvXs3\n3nhjbHnKUCiEBQsWpL2OyGsvcDtMD3ADxhZFbc52pgpdV1cXBgcHAQBz587FkiVLXKcZHBzEvffe\nCwCYOnUq/vznP/uU6+ITCoXw6KOP4ktf+hIA4I033sAXvvCFhDTLli3Dfffdh5UrVwIA3n77bVx3\n3XUAgIqKCrz00ks4fvw4A+AEDhw4kPD7unTpUs9pnn/+eUydOhWNjY25z3CWeFkMoVNEbsDYYgj9\nAFpV9VyuMleIIpEInn/+efz2t78FANx77734/e9/7zpNV1cX/vSnP7Hk58KqVatw/PhYi8vAwADW\nrVuHF154IXZu+fLlCSW+gYEBLFu2DIcPH8bAwAD27NkTK0FSepFIBM899xza29sBAHfddRf27t3r\nKU0kEkFHRwceeugh/zKeBV46QSpUtV9VN6nqGjv1zfVvloiUisizTqeJiDSl2WO44B04cADz5s2L\nHc+cOROdnZ2u0+zevRvPPPMM3nzzTX8yXMRuvvlmnDs39vd1YGAgJZiFQiFUVFTEji9cuMCA59H+\n/fsxf/782HF5eTk6Ojo8pens7ERdXV3uM5tlXjpBWtKcOyciu1xe34mxajPsUJqOzMkL05tvvony\n8vLYcXl5eaxaMFGaSCSC1atXY3BwEF/96lfR1dXlW76LUXxgA4DR0dGUc4cOHcKyZcsSrhkeHs59\n5iaR06dPY9asWbHj8vJynD9/3nWav/71r2mbgYrBuFVgu+VlB0y7X5mIJM/1LYP72R9dqrpdRJri\nzi12ndMCMTg4mBDcpk6dmhIAM6UpLS3FunXrsG7dOnR2duLJJ58syr+afhkYGEg4DoVCKedeeeUV\nVFZW4qWXXkJfXx+mTZuWUCWmiQ0ODiYEt0y/05nSvPnmm3jooYewb98+fzKcReOWAO0+wGsATIMp\nvU1L+ioBsNXtzWwvcpWIrLLziLsvM995Ex/YAODixYsp59ykaWhoQCgUyk0mJ4kTJ04kVGcrKirQ\n29ubku5nP/sZHnzwQZw4cQIDAwMpQZLGdyW/0x0dHWhoaMh5HnNlwiqwqnbDLH7QrKqfSPPlagaI\nTXcBJnB+BUC7qq65ksznw9y5cxP+Og4ODia0jbhNA5i2Qcpsz549WLBgQex42rRpeOWVVwAg1uvr\nCIVCePLJJ/Hggw/6msfJYN68eQlV3sHBQdx4442u0nR0dGDt2rW466670NnZiaeffhpPPfWUb3m/\nUq56gW1Pb9pAJyKrVHXPRO9h0+0GsNtbFgtLfX09du8e+xEikUisGtvV1YW6urqMaZxqcGlpKSKR\nCObOnet7/ovJ6OgofvrTn+L73/8+wuEwfvjDHwIwwe7nP/95QjBcvnw5HnzwwYTSX0VFBVatWhX7\nniXD9FasWJHQoTE6Oor6+noApkOvvr4+YxonHQA89dRTmDVrVlENg/GyMfrNMO2BVUkvjajqdBfX\n7wBwBqbNsF1VI24zWWgboztj/EZHR7FkyRIsWbIEkUgE99xzT2xcX7o0nZ2d+NWvfoXVq1ejvLy8\nINv/uDG6PwptY3RnjN/IyAiWLl2KpUuXIhKJ4M4778SRI0cypolXqAFwvI3RvQTAYzCrPyfrUNW1\nXjJk2wLrYMYRHpoofaEFwMmMAdAfhRYAJ7PxAqCXYTD9TrsfgG3230UAXI3lEJGbRWS5XVRhG0x7\nIMcrEFHeeJkK1y8i/wrgHACIyC9hlsVfD+A5F9f3AeiFWVq/6Do/iGjy8boe4EGYdsBNMPsCR+F+\nMYR/A/AbILaVJgA0cCtNIsoXLytCb4Jpt9tke4XrAbTBjBNMS0TO2M4TAFgHUwLsi/tKN7uEiMgX\nXjZGvxmmutsMmPGBIhKCKQmmpao3xB1usAOr49+zxlt2iYiyx0snSCeARsQNg7Hj+toyXhEnTfAr\nTT5HROQnTwHQ9gLHhq3YElytm4vtSjDL7ffHYJbTf8xTbomIsshLJ0iV3Rf4mD2ug6kSu+0E6VPV\nQyLyLIBhVa0WkVUe7k9ElFVeAuAmmODnzOB3BhducHl9mYhsBLAWQEXctRNOoyMiygUvvcD9MAug\nboZp99sGYLabmRz2+u0wPb+VqhoRkdUw7YpERHnheipcLrhdSIFT4fzDqXD+4FQ4/4w3Fc7rQOjL\nZpfC34DEKnQUwCf9ygMRUTzfAiDMEJpKVR11TnAcIBHlk5dhMFeqBcA/k86xHkBEeeMpAIrIFjuM\nxTl+2MPlGwCMisg/ReRjEfknzNQ4IqK88LItZjvMNLj4BVF7nW0uXdgCYJodTP1Ju5wWV4Uhorzx\nUgI8a4PW8bhzYZjB0G50A9hqB1M7pUfuC0lEeeMlAC6yQatSRB6z+wEfgPtFTTtgen5LAEBVn0MR\n7gtMRJOHl17gnQDaYYauNGBsJkizy+snxb7ARDR5uA6AqtopIjfAdGZUwswBbrVrA7oSvy8wzIyS\notsXmIgmjyueCSIiN6vqCZdpV8MsohAG0ON2T2HOBPEPZ4L4gzNB/JOVmSC2zS9ZGcw2l2l3hbMl\nvW5nC8zJsC8wEU0eXtoAM232mXFFaJhVoPcAgIgsT144wS6K6np/YCKibPISALuRuofHQphOkUw6\n4/YEqROR5B7jDQC+7SEPRERZ46UTpD7N6YMi8haATDu7/T8A3wcwA0A1TPtfvAVgACSiPPHSBvhW\nmtNhmHbATP6Hqq6w1z+pqj9Mes873N6fiCjbvFSBZ2c4P944wPgq8H8TkS9gbPwgAHwFwB895IGI\nKGu8BMBmAK2IC2DxS1tl0A7TblgFVoGJqMB4CYCLATSp6mfdXmAD5LcAMwbQDoOJEZEFHu5PRJRV\nnuYCA5ioxJeRqu4WkYqkc8czJCciyjkvJcD1AGpFpDTpfIuqZqzG2jbA52CquxARwGyO9LCqvuEt\nu0RE2eOlBHgAwOMALiR9ZVwOy1ZxDwHYBaAeph1wjT13yHaKEBHlhZcSYAlSN0Evwdgev+msR9I+\nIDDrCe62C6luATtBiChPxg2AItJjv+0GUKeqB9OkGW9jo/5MPcWqOiIiyQGViMg3E5UAF8Fsfp5x\nyat0QTFOeIL3n+h1IqKcmagNcOdE6/1NsCfIsN1IaWrSNaV2cyWuCUREeTPueoB28YKejAlMCW6h\nqmbc3FxEWgE0wXSY9MMMii4D0KmqaZfRSsb1AP3D9QD9wfUA/TPeeoATBcDkfXzTiY4XAO37LARQ\nA2A6TKmv28sYQAZA/zAA+oMB0D9XsiBqH8Zftr4EwIR7A6tqn30vIqKCMVEAPKuqm8ZLICJnspgf\nIiLfTNQJ0igiXx4vgaq2ZTE/RES+magNcKv9Nqqqm/3JUlpsA/TJxx9/nO8sBMJVV3mZg0BXIhqN\nXl4nSAEpikxOBgyA/mAA9M94AdDLXGAiokmFAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKL\nAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQ\niAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgC\niwGQiAKLAZCIAosBkIgCiwGQiAKLAZCIAosBkIgCiwGQiLImFArlOwueXJXvDBBR/q1evRpVVVUA\ngL6+Phw8eDAlTVNTEwBg9uzZ2LVrF44fPw4AqKysRH9/PwDgwoULCIfDPuX6yjEAXoZ9+/bh/Pnz\nAID58+fjtttuc52mra0NO3fuRCgUwjPPPIP58+f7l/EitH//fgwODgIA5s6dm/ZZZ0rT3t6O9vZ2\nXLx4EU8//TTmzZvnX8aLSCgUwubNm1FdXQ0AOHPmDG644YaENDU1NWhsbER9fT0AYHh4OBboGhoa\nUFZWhtHRUX8zng3RaLQYvgrG6Oho9Mtf/nLsuKamxnWav/zlL9G9e/dGo9FodNu2bWmvzbePPvqo\nYL6Gh4ej9913X+y4pqbGdZpz585FT548Gf3oo4+ira2t0UceeSTvP0/8F4CC+Wpqaoru2LEjdnzg\nwIFoU1NTQpqtW7dGt27dGjs+duxYtKamJgogeubMmeiOHTuiCxYsyPvPku4rOk5sYRugR3v37sWN\nN94YO541axZ27drlKk0oFMKdd94JANi4cWOshEjp7du3L6GEXF5ejvb2dldpZs2aFSvxlZeXY9as\nWf5kuggtWrQIZ8+ejR339/dj9uzZCWnKyspiVWTAlACrqqoQCoXQ2tqKqqoq9PX1YfXq1b7lOxt8\nC4Ai8paIVPh1v1w5depUwodp1qxZePvtt12lif+gnj9/ntXfCZw6dQrXXXdd7Li8vDzlj8ZEac6f\nP4+9e/fi0UcfzX2Gi1R8YAOAkZGRlHNdXV2ora1NuGZ4eBijo6PYvn076uvrsX79erS1tfmS52zx\nswS4M/mEiCz38f5ZkfwBnDp1aso5N2mOHDmCxx57LDeZnCScdj1HaWlpynOcKE17eztee+01fPOb\n38xdRouc04HhKCsrSzm3e/dubNmyBe3t7di4cSPC4TD6+voS0rS1tWF4eDjn+c0mPztB1gJoEZEL\n9rgEQAjAJ33MwxVLrkpdvHgx5dxEaSKRCN5++22sXbs2dxmdBMrLyxOOI5FIyrOdKM2jjz6KpqYm\nLFmyBJFIBKWlpbnLcJHq7e3FokWLYsdVVVVobW1NSbd9+3YApkOkv78f586dS0mTHDgLnZ8lwFYA\nswFU2a9KAN/y8f5ZMX/+/JQq1uc//3lPaXbt2oWNGzfmPrNFbv78+QnNC4ODgwltq27TlJaWYt68\neQx+GbS3t8d6gAEgHA5j9+7dAJDSphcKhdDS0oLGxkYAZgiMM/YvFAqllAoLnZ8BsB3A4wCaVXUU\nQCOAXeNfUnjuuusu/O1vf4sdj46Oxjo29u3bN2GaXbt2JZT8jhw54ke2i9Kdd96JU6dOxY5HR0ex\nYsUKAGboy0Rp4iWXFGnM6OgotmzZgo0bN2Lr1q1obm4GYAJafJve6tWrY8HPKf3V1tait7cXGzdu\nRG1tLTZt2pSXn+FylUSjUV9uJCIHAPQDuKCqm+25/aqa+tuayp9MuuSM8RsZGcEXv/hF3HbbbYhE\nIqipqUFPT0/GNPv27cP3vve9hPfq7u4uqB7Kjz/+ON9ZSOCM8RsZGcHSpUtjz7q+vh6vvfZaxjRH\njhzBE088gTVr1qC8vBxLly4tqBLgVVdxCK5fotFoSabX/AyAG1V1u4g0qWqbPTesqm6GjRdUAJzM\nCi0ATlYMgP4ZLwD6+r8gIg8DqBKRVQD+J4BuP+9PRBTPtxIgAIjIagB1AMIAelR1u8tLWQL0CUuA\n/mAJ0D8FUQV2iEh8Q8waVX3OxWUMgD5hAPQHA6B/CqIKLCJbAazHWDBzxgG6CYBERFnn55+hWgCV\ndggMAEBEany8PxFRAj/HAf5vAP9MOjfk4/2JiBLktA3QVnudoeQlAKYBGI47rlRVN1Ph2AboE7YB\n+oNtgP7JZxtgF8xsj5E0r5UAeDDH9yciyiinAVBVY+tqi8jDyT2+IvLfc3l/IqLx5HwYjIgsANAG\ns/hB/Fo5YQD9qrrYxduwCuwTVoH9wSqwf/I+DtAGwWokzfxQ1dT1dNJjAPQJA6A/GAD9M14A9KUX\nWFWPw7QHJqzVLyJcEZSI8ianf4ZE5AyABlU9AVP6S174IATgP3OZByKiTHLdCRK/t96G+E4RgAOh\niSi//BwI3SMiz4rIfwCxlWF6fLw/EVECPwNgJ8zYvxIAsENiOny8PxFRAj8DYJeqfgtmVWiHmyEw\nREQ5kc8FUTeDC6ISUR5xQVRKwHGA/uA4QP/kbSC0iFSo6kAW3ooB0CcMgP5gAPRPPgPgfpj9gNMt\nhgAAUNVDLt6KAdAnDID+YAD0Tz5ngpTABL+SuK9pMD3C7AEmorzK9Z+hxjQrQLcBOJb8GhGR3/zc\nF/hZmD1BNnno/HCwCuwTVoH9wSqwf/K6KZKI3AxT5QWAarswAhFR3uW0DVBEtgDohRkEfUNy8LPj\nAYmI8iLXvcD/hCn99SO1GjsbwGruCVJYWAX2B6vA/slnFbhRVXdnetEOjCYiygtfZ4JcgaLI5GTA\nEqA/WAL0T95XhCYiKkQMgEQUWAyARBRYDIBEFFgMgEQUWAyARBRYDIBEFFgMgEQUWAyARBRYDIBE\nFFgMgEQUWAyARBRYxbIYAhFR1rEESESBxQBIRIHFAEhEgcUASESBxQBIRIHFdbk9EJFaAF0wmzw1\nqmqfiKwH0Aqg254byWceC52IPA5gOoAhmI2xWlW1L+71MgBrADQC+A2AT9nvW1W1M/UdPd//rKrO\nvtL3KTYi0gCgA8A2VW2258rsOQDYoKr9Ga5NeWb2s1DnvFexYgnQA1XtBtAHoNP50KrqTgAjMB/Q\njMFPRBb6k8vCJSKtAKararOqbgPQDOBg0rNpAXBMVesA/Gvc991ZykZjlt6nqNg/Hp0AHneet/19\nbYH53U0b/KzYM4v7vxoG0JCj7PqGAdC7YZjSS7KJgl9bznJUBOwzWA9gi3POfgC3IPHZVMV9PyMp\n7RWLL20GUD+AnRgr9TnGfbbOM0v6PZ4UNR1WgXNARFoA9ABYq6qNAKoBVInIelXdaauBAFCH4FSb\nawGMpPlZ+2BKIU41rRrAWhH5ctz3YVv6RvKzs++7GaYZohmmdN4cl3YEwCJV3WCrba0AXgDwv+z5\nPlsy7VLVzsn+f2Ofw1kRaVXVDcmvp3m+1TDPrA5xv8ewJXKbfi2A7qTnHv8eawBsALALpqpdME0Q\nLAFenrUi0up8AShzXrB/JctslWPYfui6AQzb4NcAEwi2wfxFbsnHD5AH02F+3mTDACAiVfaZ9QPY\npar/Fve982FL9+y6ASwE0A5gEUwp02mjcpooOuz33QDCAP4LJvCGbR7O2uAXlP+bRgDrnWfkSPfz\nxz0zIO7dOW4GAAAFtElEQVT32B6HYUqUNRh77uP9H3XCBMWCwRLg5dll/4MBACKyxvneVhec0kaV\n/Yr/4NcBGLF/RSdV6WICQ0is3jqqAGCCNihHyrNT1RERiVWRbcM+ABwD0CEis9OVdGBKNRtgPpzO\n/0Mg/m9sqbcZ5g9DEy7/5x+Oe+7Os0/3HsMwQdHN/7GvGACzzP4StKlqo4ika3APw1a3fM5avnUD\naBGRsqRq5WKY0pgbrp+dDYyVMEGwV1UXJb2+U0Ra7Ae13ev7FztV3SYidTBtes7vaTZ+/pT3iPuj\nVHBYBfYuDFOdi1eGsWrwethqHRJLPE41ogemzQoAYD+Ak54tGXcirlppPxjrYUohbrh+diLSoKoj\ntge5LMOHsB1mKIcTkCf7/01yCTz5D7Sbnz+c5pzX9ygYXA3GA1ut7cDYOMD+5HGAML9kHfb4LEwD\ncQ2AgzDVsmaYv7q19njSNbSPZ7xxgLb9tBemXem3AF613zfHVbU6EPfs7PcdMO1/YZhxmo32+0aY\n/5sqW+JxxnE22+OFAGIdLOnef7L838SNA2xMKp05baVOO2vy861G4jPrta/9XwA/ROJz32BL1snv\nsQa2IyX+WRcCBkAiCixWgYkosBgAiSiwGACJKLAYAIkosBgAiSiwOBB6EoobmgOY0fjtMMMZ+mGG\nM1z2iHwRcYb5LASw05ll4QyPyDDrIvk9ar0Mh8h0z2zxmp8s3tf1M6PcYAlwErJzNZ2xXt32A9YM\ns3xR7xW+dz/iVnQBYgOaF8KM/RpX3FjKK7pntlxOfrJ0X9fPjHKHATA4jtl/y7K9NqEdLDwbZlBs\nRrYk15XNe1+JfObH7TOj3GIVODjipzD1x82KAMwE9laYObm7YBYJ6IOZQdEIxGYStMBUqY/FvZcz\nu8N5bZo9V2XP9cOUPJ1ZG4AJwh2w1XH73p7umU66e9qJ/870O2dxiiY7+8RZ0NNtftbb88dgpvCN\nwFTJm+0flRb7TOtgV1ge5zn3pHlmKfeNW7XZyW+/nd5HWcASYHA4H/ZuO0e2G2Or1NTBfBCXwHzY\nOuzabg12wQCnDa7fLiqQMD0sfmUcIFa967Xpm2GCzua4dCOq6kwldN7b0z2TZbqnDUCPw0zQb4Sp\ndjrLY3nJj7OWoNOk4KxG05x07232tV678EO657wwzTNLe1+YQFttz034HMgblgAnv4X2g1QFO1cz\nTZot9oPcAhMs6kRkNsx85jKMBU9n1ZaeCe653l7npKtD+rUAAVPiQY7v2YnE+6dblmui/DjXOKuB\n98MEU+fegJn7DZjFMJy9TeKf9xbNvIdGpvsOwZRQe2FK0W4XjiAXGAAnv75xPnTJnA95T9J6h06P\n8tnUS9JKWPF3gh7WnN/Trn23QUTcvE+m/JTBlABnx6VzgrNzzimdnU06f6X3XQsTbM/CBMp0f8To\nMrAKTPGcUtLipPPOB9vtBzp+gc283tNWXc/CrNLt5g9B2vzYTotFAGptSbkdZpUfYCzgOUtuTU86\n70am51BlmwCcEmJR78JWaFgCpHitMFXgBhs4hmHGD+6y550qnxNkMq0N57zPehHpUNVuuz6fMzSn\nzLZ5VeX6nnHXjYzT++0mP+0wS5o1pVkwdCdMh0Z86dAZf+lWpvvWiUizXWaqDGOBkLKAJcBJyH7w\nnR7X2nSLUsYt2Q/YRUrteLtGmNJIB8ymTjttj2kzgGrbFjVi00BEqmRsE5wyO6i4H2NtcB22h9Wp\nkjbb61tUtfty75n884xzz1Z7bj1MQOmECYZOG6Or/NjnWWbfO2q/esXsZeKUDp1nUQagxq5KnfKc\n7c+Q7pmluy9gOlRa7M/HAJhFXA+QyCUbiOPlZGYK+YdVYCIXbIltOH4Mni2VpdsjmooES4BELiQN\nsh6CXdY/eTwfFRcGQCIKLHaCEFFgMQASUWAxABJRYDEAElFgMQASUWAxABJRYP1/cutkaYvdNFUA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1795c3f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rc('pdf', fonttype=42)\n",
    "plt.rcParams['ps.useafm'] = True\n",
    "plt.rcParams['pdf.use14corefonts'] = True\n",
    "plt.rcParams['text.usetex'] = True\n",
    "plt.rcParams['font.serif'] = 'Times'\n",
    "plt.rcParams['font.family'] = 'serif'\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix = confusion_matrix(y,y_preds)\n",
    "matrix_proportions = np.zeros((3,3))\n",
    "for i in range(0,3):\n",
    "    matrix_proportions[i,:] = confusion_matrix[i,:]/float(confusion_matrix[i,:].sum())\n",
    "names=['Hate','Offensive','Neither']\n",
    "confusion_df = pd.DataFrame(matrix_proportions, index=names,columns=names)\n",
    "plt.figure(figsize=(5,5))\n",
    "seaborn.heatmap(confusion_df,annot=True,annot_kws={\"size\": 12},cmap='gist_gray_r',cbar=False, square=True,fmt='.2f')\n",
    "plt.ylabel(r'\\textbf{True categories}',fontsize=14)\n",
    "plt.xlabel(r'\\textbf{Predicted categories}',fontsize=14)\n",
    "plt.tick_params(labelsize=12)\n",
    "\n",
    "#Uncomment line below if you want to save the output\n",
    "#plt.savefig('confusion.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x121eb5890>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD5CAYAAADLL+UrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEDJJREFUeJzt3U9u22qWxuE3HY8CGLAsaAPK4Mwd3A0U7GGhJ7o38wZu\nsoACnFpCJ0AvIGmg5rE1KfQwTm/AuZqfQbwBI7GAAJm6Bv5Y4pUpiqTMP8n5PYBh6kiKjj4zL+lP\npPno9vZWAIAY/qPvBgAA3SH0ASAQQh8AAiH0ASAQQh8AAiH0ASCQvW0PMLNzSceS3rn7q1R7LelS\n0tTd3+xaAwB041HZcfpmduzuF2n5RtIzSUdKgZ0LcDWtuft80+tfX39rfBLBaPRENzffmz69NfRV\nz1D7kobbG33V8zP2NZnsP9p0X+n0Thb4ySdJXyU9l3SVapeSTnastWJv73Fb//RO6KueofYlDbc3\n+qonWl9bp3ckycwOJC3cfZmWs+BeSpqm5V1qhUajJzu98clkv/Fz20Rf9Qy1L2m4vdFXPZH6qhT6\nkn7L5vO1CuuFpGwDcLhDbaNdfuWaTPZ1ff2t8fPbQl/1DLUvabi90Vc9P2NfZRuLKh/kHks6S8tT\nSe+12kOfSvqQW25aAwB0oHRO38xmks4l/WFmnyUdpQ9ex+m+sbvPd6m1+u4AAH9SevRO33Y5eudn\n/JWtTfRV31B7o696fsa+Gh+9AwD4uRD6ABAIoQ8AgRD6ABBI1eP0Aaz569/+2cvr/uPvf+nldfFz\nYE8fAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIf\nAALZGvrpYuiVmdlB83YAAG0q/dPKZnYk6aOkUbr9QtKrdPehpN/dfW5mp5Jepvqz9NjXki4lTd39\nzaYaAKA7pXv67r6Q9DVX+uTuT939qaR3ki5SfZzV3X1pZjNJX9x9LmlsZrOiWgvvBwBQotacftoI\nZA5SwE8lHZnZbS7In0u6SsuXkk421AAAHWp05awU9J8lyd2vJJ1kU0FmdiHpQKuAX0rKPhcoqm00\nGj3R3t7jJi1KkiaT/cbPbRN91TPUvvpSZTyGOmb0VU8bfTW9XOJM0jxfcPeFmZ3pLsyzUF9otQE4\nLKiVurn53rC9u8G6vv7W+Pltoa96htpXn7aNx1DHjL7q2aWvso1F00M2f0l7+PekKaD3Wu3JTyV9\n2FADAHSoNPTTlM00fc9qU+X20s3s1MzO03z+W0la+7B27O7zoloL7wcAUKJ0eifttT9aq11pddim\nNh166e7ZY+ZlNQBAdzgjFwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQB\nIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIJCtoZ+uibvpvoOHbQcA0KbSa+Sm\nC6J/lDTK1U4lvUw3n6Xaa0mXkqbZNXOr1gAA3Snd008XRv+6Vh67+9P0tTSzmaQv7j6XNDazWdVa\nG28IALBZrTn9NNVzZGa3udB+LukqLV9KOqlRAwB0qHR6Z527X0k6yaZ9zOxC0oFWYb6UlH0GULW2\n0Wj0RHt7j+u0+CeTyX7j57aJvuoZal99qTIeQx0z+qqnjb5qhX7G3Rdmdqa74M4CfKHVBuCwYq3U\nzc33Ju1Juhus6+tvjZ/fFvqqZ6h99WnbeAx1zOirnl36KttYNAr9TAr/91rttU8lfcgtV6kBADpS\nOqefpnGm6bvM7NTMztN8/ltJWvtgduzu86q1Nt8YAOC+0j39dPTOo9ztwsMs3f1VWpzXrQEAusMZ\nuQAQCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQ\nCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIFsDX0zm257zNrjD5q3AwBoU+k1ctMF0T9KGuVq55KOJb3L\nrnlrZqeSXqaHPEu115IuJU2za+sW1QAA3Snd008XRv+a3TazY3f/1d1Hkl7kfgsYu/vT9LU0s5mk\nL+4+lzQ2s1lRraX3BADYoNacvrtf5G5+kvQ1Bf+Rmd3mgvy5pKu0fCnpZEMNANCh0umdTdK8/cLd\nl5KWkk6yqSAzu5B0oFXALyVlvxEU1TYajZ5ob+9xkxYlSZPJfuPntom+6hlqX32pMh5DHTP6qqeN\nvhqFvqTfsvn8jLsvzOxMd2GehfpCqw3AYUGt1M3N94bt3Q3W9fW3xs9vC33VM9S++rRtPIY6ZvRV\nzy59lW0sah+yaWbHks7S8r299fQ5wHut9uSnkj5sqAEAOlQa+mnKZpq+K83Zn0v6w8w+624u/9TM\nztN9byVp7cPasbvPi2otvi8AQIHS6Z201/4od3suqVJY56Z/5mU1AEB3OCMXAAIh9AEgEEIfAAIh\n9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEg\nEEIfAAIh9AEgkK2hX3TxcwDAj6n0GrnpgugfJY1ytdeSLiVN3f3NrjUAQHdK9/TThdG/ZrfNbCbp\nS7pA+tjMZrvUWntXAIBCdef0n0u6SsuXkk52rAEAOlQ6vVPgQKvgXkrK5vt3qW00Gj3R3t7jmi2u\nTCb7jZ/bJvqqZ6h99aXKeAx1zOirnjb6qhv6WVgvtNoAHO5QK3Vz871meyuTyb6ur781fn5b6Kue\nofbVp23jMdQxo696dumrbGNRN/Tfa7WHPpX0IbfctAYA6EjpnH46emeavmvtQ9ixu893qbX6zgAA\n95Tu6aejdx6t1V6lxflD1AAA3eGMXAAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAI\nfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEBaCX0zO2jj3wUA7Kb0\nGrlFzOyFpOxat4eSfnf3uZmdSnqZ6s/SY19LupQ0dfc3m2oAgG402dP/5O5P3f2ppHeSLlJ9nNXd\nfWlmM0lf3H0uaWxms6Law7wNAEAVtUPf3Re5mwcp4KeSjszsNhfkzyVdpeVLSScbagCAjtSe3smk\noP8sSe5+JenEzI4kfTSzC0kHWgX8UtI0LRfVCo1GT7S397hpi5pM9hs/t030Vc9Q++pLlfEY6pjR\nVz1t9NU49CXNJM3zBXdfmNmZ7sI8C/WFVhuAw4LaRjc33xs3N5ns6/r6W+Pnt4W+6hlqX33aNh5D\nHTP6qmeXvso2FrscvfNL2sO/J00BvddqT34q6cOGGgCgI41CP03tXOVun5rZeZrPfytJax/Wjt19\nXlTb/S0AAKpqNL2T9vBf5W4XHnrp7tlj5mU1AEA3OCMXAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEg\nEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgkJ1D\n38wOHqIRAED7Gl0j18xOJb1MN5+l2mtJl5Km2TVzq9YAAN1ouqc/dven6WtpZjNJX9x9LmlsZrOq\ntQd6HwCACmqHvplNJR2Z2W0utJ9LukrLl5JOatQAAB2pPb3j7leSTszsSNJHM7uQdKBVmC8lTdNy\n1Vqh0eiJ9vYe123x3yaT/cbPbRN91TPUvvpSZTyGOmb0VU8bfTWa05ckd1+Y2ZnugjsL8IVWG4DD\nirWNbm6+N21Pk8m+rq+/NX5+W+irnqH21adt4zHUMaOvenbpq2xj0Tj0Myn832u11z6V9CG3XKUG\nAOhAkzn9UzM7T/P5byVp7YPZsbvPq9Ye8L0AALZoMqdfeJilu79Ki/O6NQBANzgjFwACIfQBIBBC\nHwACIfQBIJCdD9kEgJ/Vf/33//f22v/3P//Zyr/Lnj4ABELoA0AghD4ABELoA0AghD4ABELoA0Ag\nhD4ABELoA0AghD4ABELoA0AghD4ABELoA0AghD4ABNJK6JvZQRv/LgBgN43+tLKZnUs6lvQuu+at\nmZ1Kepke8izVXku6lDTNrq1bVAMAdKP2nr6ZHbv7r+4+kvTCzKbprrG7P01fSzObSfri7nNJYzOb\nFdUe7q0AALapHfrufpG7+UnS1xT8R2Z2mwvy55Ku0vKlpJMNNQBARxpfOSvN2y/cfSlpKenEzI4k\nfTSzC0kHWgX8UlL2G0FRrdBo9ER7e4+btqjJZL/xc9tEX/UMta++VBmPoY4ZfdXTRl+7XC7xt2w+\nP+PuCzM7012YZ6G+0GoDcFhQ2+jm5nvj5iaTfV1ff2v8/LbQVz1D7atP28ZjqGNGX/U17atsY9Ho\n6B0zO5Z0lpbv7a27+0LSe6325KeSPmyoAQA60uSD3Jmkc0l/mNln3c3ln5rZebrvrSStfVg7dvd5\nUe3h3goAYJva0zspqCuFdW76Z15WAwB0gzNyASAQQh8AAiH0ASAQQh8AAiH0ASAQQh8AAtnljNxB\n++vf/tnba//j73/p7bUBoAx7+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQCKEPAIEQ+gAQ\nCKEPAIEQ+gAQCKEPAIF0/gfXzOy1pEtJU3d/0/XrA0Bkne7pm9lM0pd0cfVxug0A6EjX0zvPJV2l\n5UtJJx2/PgCE9uj29razFzOzD5JeufvCzI7TMsEPAB3pek9/KWmalg+02usHAHSg69B/r1XoTyV9\n6Pj1ASC0Tqd3pD8dvfOLu7/q9MUBILjOQx8A0B9OzgKAQAj9ATKzA3qorq1ezWy6/VHd26WvNn+u\nVfvqet0a6nj1pfMzch/KtjN7i+7v6mzgCr2dSzqW9C77XMPMTiW9TA951lNf93roYszKXsPMXkjK\nPvs5lPS7u8/bHi8zO5L0UdKoas8djdW2vvpat7b11de6tbGvvtat3Ovf+1mt3d/KOvZD7ulvO7O3\n6P6uzgau0Nuxu//q7iNJL3J7IWN3f5q+ll33VdRDF2NW4TU+ZT1JeifpoqjXh+7L3ReSvlbtuav1\na0tfvaxb2/oq6mEI46We1i2p9GeV3d/aOvZDhr62n9lbdH9XZwOXvo67X+RufpL0Nf3Aj8zstsU/\nTVHa14YeuhizbeO1yN08SIHRxXiV6XP92qjHdatUj+tWqT7XraKf1dpDWlvHftTQz5/YlT/hq+z+\nbc/pqjdJ/54rXLj70t2v0pnJzyT9b0vziKV9beihizGrOl5TSZ9Leu1Sn+vXVj2sW6V6XLcq6XPd\nyv+s1u5qbR37UUN/25m9Rfd3dTZw1df5bX0eL+15nKmd/wCV+lrroYsxq/oaM0nzfKHl8SrT5/pV\nRdfrViU9rFtV9blu3ftZJa2tYz9q6Bee2ZvbKhfd39XZwNt6U/q7Q2dp+d5KtfZrZ2d9FfTQxZhV\n7esXdy/bULVuIOtXWV99rVtb+yroYRDjlfSybhX9rLpYx37I0F/7IGOcbkt3n9IX3l/ynE57S/Vz\nSX+Y2WfdzR+emtl5uu9tT33d66GLMdvWV+ptqtxeTRfjlY76mKbvmd7Xr7K++lq3KvTVy7q1ra90\nf+frVnqdez+rfG9trmOckQsAgfyQe/oAgGYIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAI\n5F8SgEh03ntyyQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11ead58d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#True distribution\n",
    "y.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x123f48a90>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD3CAYAAADxJYRbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD0hJREFUeJzt3T9uG1m2x/Gfx4wMCBBNcANUcHIZvYGBFQ5ewm7nA7S9\ngAGkWcKzgbcA+wGTS2IymFDybEBu5iewNiDYImDAqSbQrSaHKhariqo/7vP9AIaKp0jz8Kr849Ut\nlvzk7u5OAIAY/tR1AwCA9hD6ABAIoQ8AgRD6ABAIoQ8AgQy6bqDIzc232h8tGg6f6fb2+2O28yjo\nq5q+9iX1tzf6quaP2Nd4vPdk074/7Ex/MHjadQu56KuavvYl9bc3+qomWl9/2NAHADxE6ANAIIQ+\nAARC6ANAIIQ+AARC6ANAIIQ+AARC6ANAIIQ+AATS61/DAPTZX/72z06e9x9//3Mnz4s/hq0zfTOb\nVPkLzWy/fjsAgCYVzvTN7FDSR0nDdPu1pJO0+7mkX919ZmbHkt6k+ot037eSriRN3P3dphoAoD2F\nM313n0v6ulL65O4H7n4g6YOky1QfZXV3X5jZVNIXd59JGpnZNK/WwOsBABSodCI3vQlk9lPATyQd\nmtndSpC/knSdtq8kHW2oAQBaVOtEbgr6z5Lk7teSjrKlIDO7lLSvZcAvJGXnBfJqGw2Hz3b69aLj\n8V7txzaJvqrpa19dKTMefR0z+qqmib7qfnpnKmm2WnD3uZmd6T7Ms1Cfa/kG8DynVmiX/9hgPN7T\nzc232o9vCn1V09e+urRtPPo6ZvRVzS59Fb1Z1P2c/k9phv9AWgI61XImP5F0saEGAGhRYeinJZtJ\n+prVJlqZpZvZsZmdp/X895K0drJ25O6zvFoDrwcAUKBweSfN2p+s1a61/NimNn300t2z+8yKagCA\n9vBrGAAgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh\n9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAALZGvpmNinYt/+47QAAmjQo2mlmh5I+\nShqu1I4lvUk3X6TaW0lXkibu/q5KDQDQnsKZvrvPJX1dK4/c/SD9WZjZVNIXd59JGpnZtGytiRcE\nANis0pp+Wuo5NLO7ldB+Jek6bV9JOqpQAwC0qHB5Z527X0s6ypZ9zOxS0r6WYb6QlJ0DKFvbaDh8\npsHgaZUW/8t4vFf7sU2ir2r62ldXyoxHX8eMvqppoq9KoZ9x97mZnek+uLMAn2v5BvC8ZK3Q7e33\nOu1Juh+sm5tvtR/fFPqqpq99dWnbePR1zOirml36KnqzqBX6mRT+p1rO2ieSLla2y9QAAC0pXNNP\nyziT9FVmdmxm52k9/70krZ2YHbn7rGytyRcGAHiocKafPr3zZOV27scs3f0kbc6q1gAA7eGKXAAI\nhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAH\ngEAIfQAIhNAHgEAIfQAIhNAHgEAIfQAIhNAHgEC2hr6ZTar8hWa2X78dAECTBkU7zexQ0kdJw5Xa\nuaSXkj64+0mqHUt6k+7yItXeSrqSNHH3d5tqAID2FM703X0u6Wt228xeuvvP7j6U9Hrlp4CRux+k\nPwszm0r64u4zSSMzm+bVGnpNAIANKq3pu/vlys1Pkr6m4D80s7uVIH8l6TptX0k62lADALSocHln\nk7RuP3f3haSFpKNsKcjMLiXtaxnwC0nZTwR5tY2Gw2caDJ7WaVGSNB7v1X5sk+irmr721ZUy49HX\nMaOvaproq1boS/olW8/PuPvczM50H+ZZqM+1fAN4nlMrdHv7vWZ794N1c/Ot9uObQl/V9LWvLm0b\nj76OGX1Vs0tfRW8WlT+yaWYvJZ2l7Qez9XQe4FTLmfxE0sWGGgCgRYWhn5ZsJumr0pr9uaTfzOyz\n7tfyj83sPO17L0lrJ2tH7j7LqzX4ugAAOQqXd9Ks/cnK7ZmkUmG9svwzK6oBANrDFbkAEAihDwCB\nEPoAEAihDwCBEPoAEAihDwCBEPoAEAihDwCBEPoAEAihDwCBEPoAEAihDwCBEPoAEAihDwCBEPoA\nEAihDwCBEPoAEAihDwCBEPoAEAihDwCBbA19M5u00QgAoHmDop1mdijpo6ThSu2tpCtJE3d/t2sN\nANCewpm+u88lfc1um9lU0hd3n0kamdl0l1pjrwoAkKtwpp/jlaTTtH0l6UjS8x1qs6InGw6faTB4\nWrHFpfF4r/Zjm0Rf1fS1r66UGY++jhl9VdNEX1VDf1/SddpeSMrW+3epbXR7+71ie0vj8Z5ubr7V\nfnxT6KuavvbVpW3j0dcxo69qdumr6M2i6qd3VsM6ewPYpQYAaFHVmf6plsE9kXSxsl23BgBoSeFM\nP316Z5K+au0k7MjdZ7vUGn1lAIAHCmf66dM7T9ZqJ2lz9hg1AEB7uCIXAAIh9AEgEEIfAAIh9AEg\nEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIfAAIh9AEgEEIf\nAAIh9AEgEEIfAAIh9AEgkEZC38z2m/h7AQC7GVR9gJm9lnSSbj6X9Ku7z8zsWNKbVH+R7vtW0pWk\nibu/21QDALSjzkz/k7sfuPuBpA+SLlN9lNXdfWFmU0lf3H0maWRm07za47wMAEAZlWf67j5fubmf\nAn4i6dDM7iT9nEL9laTTdL8rSUe6/8lgvTbb9FzD4TMNBk+rtvi78Xiv9mObRF/V9LWvrpQZj76O\nGX1V00RflUM/k4L+syS5+7WkIzM7lPTRzC4l7Uu6TndfSJqk7bxartvb73Xb03i8p5ubb7Uf3xT6\nqqavfXVp23j0dczoq5pd+ip6s9jlRO5Ua7P09FPAme7DfDXUszeAvBoAoCW7hP5PaYb/QAr/Uy0D\nfiLpYkMNANCSWqGflnauV24fm9l5OjH7XpLWTtaO3H2WV9v9JQAAyqq1pp9m+Ccrt3M/eunu2X1m\nRTUAQDu4IhcAAiH0ASAQQh8AAiH0ASAQQh8AAiH0ASAQQh8AAiH0ASAQQh8AAiH0ASAQQh8AAiH0\nASAQQh8AAiH0ASAQQh8AAiH0ASAQQh8AAiH0ASAQQh8AAtk59M1s/zEaAQA0r9Z/jG5mx5LepJsv\nUu2tpCtJk+w/Si9bAwC0o+5Mf+TuB+nPwsymkr64+0zSyMymZWuP9DoAACVUDn0zm0g6NLO7ldB+\nJek6bV9JOqpQAwC0pPLyjrtfSzoys0NJH83sUtK+lmG+kDRJ22VruYbDZxoMnlZt8Xfj8V7txzaJ\nvqrpa19dKTMefR0z+qqmib5qrelLkrvPzexM98GdBfhcyzeA5yVrG93efq/bnsbjPd3cfKv9+KbQ\nVzV97atL28ajr2NGX9Xs0lfRm0Xt0M+k8D/VctY+kXSxsl2mBgC989f//Xdnz/2v//ufRv7eOmv6\nx2Z2ntbz30vS2onZkbvPytYe8bUAALaos6af+zFLdz9Jm7OqNQBAO7giFwACIfQBIBBCHwACIfQB\nIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBCHwACIfQBIBBC\nHwACIfQBIBBCHwACIfQBIJBGQt/M9pv4ewEAuxnUeZCZnUt6KemDu5+k2rGkN+kuL1LtraQrSRN3\nf7epBgBoR+WZvpm9dPef3X0o6bWZTdKukbsfpD8LM5tK+uLuM0kjM5vm1R7vpQAAtqkc+u5+uXLz\nk6SvKfgPzexuJchfSbpO21eSjjbUAAAtqbW8I/2+bj9394WkhaQjMzuU9NHMLiXtaxnwC0nZTwR5\ntVzD4TMNBk/rtqjxeK/2Y5tEX9X0ta+ulBmPvo4ZfVXTRF+1Q1/SL9l6fsbd52Z2pvswz0J9ruUb\nwPOc2ka3t99rNzce7+nm5lvtxzeFvqrpa19d2jYefR0z+qqubl9Fbxa1Pr1jZi8lnaXtB7N1d59L\nOtVyJj+RdLGhBgBoSZ0TuVNJ55J+M7PPul/LPzaz87TvvSStnawdufssr/Z4LwUAsE3l5Z0U1KXC\nemX5Z1ZUAwC0gytyASCQXU7k9tpf/vbPzp77H3//c2fPDQBFmOkDQCCEPgAEQugDQCCEPgAEQugD\nQCCEPgAEQugDQCCEPgAEQugDQCCEPgAEQugDQCCEPgAEQugDQCCEPgAEQugDQCCEPgAEQugDQCCE\nPgAEQugDQCCt/x+5ZvZW0pWkibu/a/v5ASCyVmf6ZjaV9MXdZ5JG6TYAoCVtL++8knSdtq8kHbX8\n/AAQ2pO7u7vWnszMLiSduPvczF6mbYIfAFrS9kx/IWmStve1nPUDAFrQduifahn6E0kXLT8/AITW\n6vKO9F+f3vnJ3U9afXIACK710AcAdIeLswAgEEK/h8xsnx7Ka6pXM5tsv1f7dumrye9r2b7aPrb6\nOl5daf2K3Mey7crevP1tXQ1cordzSS8lfcjOa5jZsaQ36S4vOurrQQ9tjFnRc5jZa0nZuZ/nkn51\n91nT42Vmh5I+ShqW7bmlsdrWV1fH1ra+ujq2NvbV1bG18vwPvldr+xs5xn7Imf62K3vz9rd1NXCJ\n3l66+8/uPpT0emUWMnL3g/Rn0XZfeT20MWYlnuNT1pOkD5Iu83p97L7cfS7pa9me2zq+tvTVybG1\nra+8HvowXuro2JIKv1fZ/saOsR8y9LX9yt68/W1dDVz4PO5+uXLzk6Sv6Rt+aGZ3Df5qisK+NvTQ\nxphtG6/5ys39FBhtjFeRLo+vjTo8tgp1eGwV6vLYyvterd2lsWPsRw391Qu7Vi/4Ktq/7TFt9Sbp\n97XCubsv3P06XZn8QtL/N7SOWNjXhh7aGLOy4zWR9Lmg1zZ1eXxt1cGxVajDY6uULo+t1e/V2q7G\njrEfNfS3Xdmbt7+tq4HLPs8v6+t4aeZxpmb+AZTqa62HNsas7HNMJc1WCw2PV5Euj68y2j62Sung\n2Cqry2PrwfcqaewY+1FDP/fK3pV35bz9bV0NvK03pd87dJa2HxxUaz92ttZXTg9tjFnZvn5y96I3\nqsb15Pgq6qurY2trXzk99GK8kk6OrbzvVRvH2A8Z+msnMkbptnR/lj53f8FjWu0t1c8l/WZmn3W/\nfnhsZudp3/uO+nrQQxtjtq2v1NtEK7OaNsYrfepjkr5mOj++ivrq6tgq0Vcnx9a2vtL+1o+t9DwP\nvlervTV5jHFFLgAE8kPO9AEA9RD6ABAIoQ8AgRD6ABAIoQ8AgRD6ABAIoQ8AgRD6ABDIfwD5A34y\n+yLEdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x123fe09d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(y_preds).hist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
